# =============================================================================
# SETUP: Import Libraries and Load Data
# =============================================================================
# This cell sets up all the tools we need for spatial sentiment analysis

# Force reload to pick up any changes to data_cleaning_utils
# This ensures we get the latest version of our custom functions
import importlib
import sys
if 'data_cleaning_utils' in sys.modules:
    importlib.reload(sys.modules['data_cleaning_utils'])

# Core data analysis library - like Excel but for Python
import pandas as pd

# Import our custom functions for cleaning and analyzing location data
from data_cleaning_utils import (
    clean_institution_dataframe,      # Standardizes and cleans location data
    get_data_type_summary,            # Shows what types of data we have
    get_null_value_summary,           # Identifies missing data
    create_location_counts,           # Counts how often places are mentioned
    create_location_sentiment,        # Calculates average emotions by location
    create_time_animation_data,       # Prepares data for animated time series
)

# Interactive plotting library - creates maps and charts
import plotly.express as px
import plotly.graph_objects as go
import plotly.io as pio
import plotly.offline as pyo

# =============================================================================
# CONFIGURE PLOTLY FOR HTML EXPORT
# =============================================================================
# Configure Plotly for optimal HTML export compatibility

# Method 1: Set renderer for HTML export (use 'notebook' for Jupyter environments)
pio.renderers.default = "notebook"

# Method 2: Configure Plotly for offline use (embeds JavaScript in HTML)
pyo.init_notebook_mode(connected=False)  # False = fully offline, no external dependencies

# Method 3: Set template for clean HTML appearance
pio.templates.default = "plotly_white"

# Method 4: Configure Plotly to include plotly.js in HTML exports
import plotly
plotly.offline.init_notebook_mode(connected=False)



# Load the cleaned JMU Reddit data (already processed and ready to use)
# This contains: posts, locations, coordinates, sentiment scores, and dates
df_jmu = pd.read_pickle("assets/data/jmu_reddit_geoparsed_clean.pickle")

# =============================================================================
# LOAD YOUR INSTITUTION'S DATA
# =============================================================================
# Replace the group number and institution name with your assigned data

# 📝 TO DO: Update these paths for your group
# Replace "group_6" with your group number (e.g., "group_1", "group_2", etc.)
# Replace "UNC_processed.csv" with your institution's file name
df_institution = pd.read_csv("group_data_packets/group_6/python/UNC_processed.csv")

# =============================================================================
# CREATE RAW LOCATION MAP (Before Cleaning)
# =============================================================================
# This shows the "messy" data before we fix location errors
# You'll see why data cleaning is essential!

# STEP 1: Count how many times each place is mentioned
# Group identical place names together and count occurrences
place_counts = df_institution.groupby('place').agg({
    'place': 'count',           # Count how many times each place appears
    'latitude': 'first',        # Take the first latitude coordinate for each place
    'longitude': 'first',       # Take the first longitude coordinate for each place
    'place_type': 'first'       # Take the first place type classification
}).rename(columns={'place': 'count'})  # Rename the count column for clarity

# STEP 2: Prepare data for mapping
# Reset index makes 'place' a regular column instead of an index
place_counts = place_counts.reset_index()

# Remove any places that don't have valid coordinates (latitude/longitude)
# This prevents errors when trying to plot points on the map
place_counts = place_counts.dropna(subset=['latitude', 'longitude'])

# STEP 3: Create interactive scatter map
# Each dot represents a place, size = how often it's mentioned
fig = px.scatter_map(
    place_counts,                    # Our prepared data
    lat='latitude',                  # Y-coordinate (north-south position)
    lon='longitude',                 # X-coordinate (east-west position)
    size='count',                    # Bigger dots = more mentions
    hover_name='place',              # Show place name when hovering
    hover_data={                     # Additional info in hover tooltip
        'count': True,               # Show mention count
        'place_type': True,          # Show what type of place it is
        'latitude': ':.4f',          # Show coordinates with 4 decimal places
        'longitude': ':.4f'
    },
    size_max=25,                     # Maximum dot size on map
    zoom=4,                          # How zoomed in the map starts (higher = closer)
    title='Raw Location Data: Places Mentioned in UNC Reddit Posts',
    center=dict(lat=35.5, lon=-80)   # Center map on North Carolina for UNC
)

# STEP 4: Customize map appearance
fig.update_layout(
    map_style="carto-positron",      # Clean, light map style
    width=800,                       # Map width in pixels
    height=600,                      # Map height in pixels
    title_font_size=16,              # Title text size
    title_x=0.5                      # Center the title
)


# Configure for HTML export compatibility
fig.show(config={'displayModeBar': True, 'displaylogo': False})

df_institution_cleaned = pd.read_csv('group_data_packets/group_6/python/UNC_processed_clean.csv')

# =============================================================================
# LOAD CLEANED DATA
# =============================================================================
# Load the CSV file you manually cleaned in Google Sheets

# 📝 TO DO: Update these paths for your group
# Replace "group_6" with your group number
# Replace "UNC_processed_clean.csv" with your institution's cleaned file
df_institution_cleaned = pd.read_csv(
    "group_data_packets/group_6/python/UNC_processed_clean.csv"
)

# =============================================================================
# APPLY DATA CLEANING FUNCTIONS
# =============================================================================
# Use our custom function to standardize the cleaned data

# Apply the cleaning function to standardize data types and handle missing values
# This function ensures all datasets have the same format for consistent analysis

df_institution_cleaned = clean_institution_dataframe(df_institution_cleaned)

# Display first few rows to verify the cleaning worked properly
# This shows the structure and sample content of your cleaned data
df_institution_cleaned.head()

DataFrame cleaned successfully!

zoom=2, # Set the zoom level here
center=dict(lat=38.5, lon=-106),  # Set the center of your view here

# =============================================================================
# CREATE CLEANED LOCATION MAP (After Manual Corrections)
# =============================================================================
# This map shows your data AFTER you fixed the location errors
# Compare this to the raw map above to see the improvement!

# STEP 1: Count occurrences using CLEANED/CORRECTED location data
# Now we use 'revised_place' instead of 'place' - these are your corrections!
place_counts = (
    df_institution_cleaned.groupby("revised_place")  # Group by corrected place names
    .agg(
        {
            "revised_place": "count",        # Count mentions of each corrected place
            "revised_latitude": "first",     # Use corrected latitude coordinates
            "revised_longitude": "first",    # Use corrected longitude coordinates
            "place_type": "first",           # Keep place type classification
        }
    )
    .rename(columns={"revised_place": "count"})  # Rename count column for clarity
)

# STEP 2: Prepare data for mapping
place_counts = place_counts.reset_index()  # Make 'revised_place' a regular column

# Remove places without valid corrected coordinates
place_counts = place_counts.dropna(subset=["revised_latitude", "revised_longitude"])

# STEP 3: Create the cleaned location map
fig = px.scatter_map(
    place_counts,
    lat="revised_latitude",          # Use corrected Y-coordinates
    lon="revised_longitude",         # Use corrected X-coordinates
    size="count",                    # Dot size = mention frequency
    hover_name="revised_place",      # Show corrected place name on hover
    hover_data={
        "count": True,               # Show how many mentions
        "place_type": True,          # Show place category
        "revised_latitude": ":.4f",   # Show corrected coordinates
        "revised_longitude": ":.4f",
    },
    size_max=25,                     # Maximum dot size
    title="Cleaned Location Data: Places Mentioned in UNC Reddit Posts",
    zoom=2,                          # 📝 TO DO: Adjust zoom level for your region
    center=dict(lat=38.5, lon=-106), # 📝 TO DO: Adjust center coordinates
)

# STEP 4: Customize map appearance
fig.update_layout(
    map_style="carto-positron",      # Clean, readable map style
    width=800, 
    height=600, 
    title_font_size=16, 
    title_x=0.5
)


# Display with HTML export configuration
fig.show(config={'displayModeBar': True, 'displaylogo': False})

create_location_counts(
    df_institution_cleaned, 
    minimum_count=2, 
    place_type_filter=["State", "City", "Country"]
)

# =============================================================================
# JMU SPATIAL DISTRIBUTION MAP
# =============================================================================
# Create a filtered map showing only certain types of places for JMU

# STEP 1: Use custom function to filter and count JMU locations
# This function applies the same filtering to both datasets for fair comparison
JMU_filtered_locations = create_location_counts(
    df_jmu,                          # JMU Reddit data
    minimum_count=2,                 # Only show places mentioned 2+ times
    place_type_filter=['State', 'City', 'Country']  # Only these place types
)


# STEP 2: Create colored scatter map
# Each place type gets a different color to show spatial patterns
fig = px.scatter_map(
    JMU_filtered_locations,
    lat="revised_latitude",
    lon="revised_longitude", 
    size="count",                    # Dot size = mention frequency
    color="place_type",              # Different colors for different place types
    hover_name="revised_place",
    hover_data={
        "count": True,
        "place_type": True,
        "revised_latitude": ":.4f",
        "revised_longitude": ":.4f",
    },
    size_max=25,
    zoom=2,                          # 📝 TO DO: Adjust to highlight interesting patterns
    title="Cleaned Location Data: Places Mentioned in JMU Reddit Posts",
    center=dict(lat=38.5, lon=-106), # 📝 TO DO: Center on area of interest
    color_discrete_sequence=px.colors.qualitative.Plotly  # Categorical color palette
)

# STEP 3: Customize layout
fig.update_layout(
    map_style="carto-positron", 
    width=800, 
    height=600, 
    title_font_size=16, 
    title_x=0.5
)


# Display with HTML export configuration
fig.show(config={'displayModeBar': True, 'displaylogo': False})

# =============================================================================
# YOUR INSTITUTION'S SPATIAL DISTRIBUTION MAP
# =============================================================================
# Create a comparable map for your institution using identical filtering

# STEP 1: Apply the same filtering to your institution's data
# Using identical parameters ensures fair comparison with JMU
institution_filtered_locations = create_location_counts(
    df_institution_cleaned,          # Your cleaned institution data
    minimum_count=2,                 # Same minimum as JMU map
    place_type_filter=["State", "City", "Country"]  # Same place types as JMU
)


# STEP 2: Create matching visualization
# Keep all settings the same as JMU map for direct comparison
fig_institution_cleaned = px.scatter_map(
    institution_filtered_locations,
    lat="revised_latitude",
    lon="revised_longitude",
    size="count",
    color="place_type",              # Same color coding as JMU map
    hover_name="revised_place",
    hover_data={
        "count": True,
        "place_type": True,
        "revised_latitude": ":.4f",
        "revised_longitude": ":.4f",
    },
    size_max=25,                     # Same size scale as JMU
    zoom=2,                          # 📝 TO DO: Adjust for your region
    title="Cleaned Location Data: Places Mentioned in UNC Reddit Posts",  # 📝 TO DO: Update institution name
    center=dict(lat=38.5, lon=-106), # 📝 TO DO: Center on your region
    color_discrete_sequence=px.colors.qualitative.Plotly,  # Same colors as JMU
)

# STEP 3: Apply identical layout settings
fig_institution_cleaned.update_layout(
    map_style="carto-positron",      # Same style as JMU map
    width=800, 
    height=600, 
    title_font_size=16, 
    title_x=0.5
)



# Display with HTML export configuration
fig_institution_cleaned.show(config={'displayModeBar': True, 'displaylogo': False})

create_location_sentiment(
    df_jmu,
    minimum_count=2,
    place_type_filter=None  # Include all place types
)

# =============================================================================
# JMU SENTIMENT ANALYSIS MAP
# =============================================================================
# Shows the EMOTIONAL tone of how JMU students talk about different places
# Red = negative emotions, Green = positive emotions

# STEP 1: Calculate average sentiment scores by location
# This function groups identical locations and averages their sentiment scores
df_jmu_sentiment = create_location_sentiment(
    df_jmu,                          # JMU Reddit data with sentiment scores
    minimum_count=2,                 # Only places mentioned 2+ times (for reliability)
    place_type_filter=None           # Include all place types for comprehensive view
)


# STEP 2: Create sentiment visualization map
# Color represents emotional tone: Green = positive, Red = negative, Yellow = neutral
fig_sentiment = px.scatter_map(
    df_jmu_sentiment,
    lat="revised_latitude",
    lon="revised_longitude",
    size="count",                    # Larger dots = more mentions (more reliable sentiment)
    color="avg_sentiment",           # Color intensity = emotional tone
    color_continuous_scale="RdYlGn", # Red-Yellow-Green scale (Red=negative, Green=positive)
    hover_name="revised_place",
    hover_data={
        "count": True,               # How many posts contributed to this sentiment
        "avg_sentiment": ":.3f",     # Average sentiment score (3 decimal places)
        "place_type": True,
        "revised_latitude": ":.4f",
        "revised_longitude": ":.4f",
    },
    size_max=25,
    zoom=2,                          # 📝 TO DO: Adjust to focus on interesting patterns
    title="Average Sentiment by Location in JMU Reddit Posts",
    center=dict(lat=38.5, lon=-86),  # 📝 TO DO: Center on region of interest
)

# STEP 3: Customize layout for sentiment analysis
fig_sentiment.update_layout(
    map_style="carto-positron",      # Clean background to highlight sentiment colors
    width=800, 
    height=600, 
    title_font_size=16, 
    title_x=0.5
)


# Display with HTML export configuration
fig_sentiment.show(config={'displayModeBar': True, 'displaylogo': False})

# =============================================================================
# YOUR INSTITUTION'S SENTIMENT ANALYSIS MAP
# =============================================================================
# Compare emotional patterns between your institution and JMU

# STEP 1: Calculate sentiment for your institution using identical methods
institution_sentiment = create_location_sentiment(
    df_institution_cleaned,          # Your cleaned institution data
    minimum_count=2,                 # Same minimum as JMU (ensures fair comparison)
    place_type_filter=None           # Same filter as JMU (include all place types)
)


# STEP 2: Create matching sentiment visualization
# Use identical settings to JMU map for direct comparison
fig_institution_sentiment = px.scatter_map(
    institution_sentiment,
    lat="revised_latitude",
    lon="revised_longitude",
    size="count",
    color="avg_sentiment",
    color_continuous_scale="RdYlGn", # Same color scale as JMU map
    hover_name="revised_place",
    hover_data={
        "count": True,
        "avg_sentiment": ":.3f",
        "place_type": True,
        "revised_latitude": ":.4f",
        "revised_longitude": ":.4f",
    },
    size_max=25,                     # Same size scale as JMU
    zoom=2,                          # 📝 TO DO: Adjust for your region
    title="Average Sentiment by Location in UNC Reddit Posts",  # 📝 TO DO: Update institution name
    center=dict(lat=35.5, lon=-80),  # 📝 TO DO: Center on your institution's region
)

# STEP 3: Apply identical layout for comparison
fig_institution_sentiment.update_layout(
    map_style="carto-positron",      # Same background as JMU map
    width=800, 
    height=600, 
    title_font_size=16, 
    title_x=0.5
)



# Display with HTML export configuration
fig_institution_sentiment.show(config={'displayModeBar': True, 'displaylogo': False})

fig_animated.layout.updatemenus[0].buttons[0].args[1]["frame"]["duration"] = 800
fig_animated.layout.updatemenus[0].buttons[0].args[1]["transition"]["duration"] = 300

# =============================================================================
# ANIMATED TIME SERIES: SENTIMENT CHANGES OVER TIME
# =============================================================================
# Watch how places accumulate mentions and sentiment changes over time
# This reveals temporal patterns in student discussions

# STEP 1: Prepare animation data with rolling averages
# This function creates monthly frames showing cumulative growth and sentiment trends
institution_animation = create_time_animation_data(
    df_institution_cleaned,          # Your cleaned institution data
    window_months=3,                 # 3-month rolling average (smooths out noise)
    minimum_count=2,                 # Only places with 2+ total mentions
    place_type_filter=None           # Include all place types (📝 TO DO: experiment with filtering)
)

# STEP 2: Create animated scatter map
# Each frame represents one month, showing cumulative mentions and current sentiment
fig_animated = px.scatter_map(
    institution_animation,
    lat="revised_latitude",
    lon="revised_longitude",
    size="cumulative_count",         # Dot size = total mentions up to this point in time
    color="rolling_avg_sentiment",   # Color = 3-month average sentiment (smoother than daily)
    animation_frame="month",         # Each frame = one month of data
    animation_group="revised_place", # Keep same places connected across frames
    hover_name="revised_place",
    hover_data={
        "cumulative_count": True,    # Total mentions so far
        "rolling_avg_sentiment": ":.3f", # Smoothed sentiment score
        "place_type": True,
        "revised_latitude": ":.4f",
        "revised_longitude": ":.4f"
    },
    color_continuous_scale="RdYlGn", # Same sentiment colors as static maps
    size_max=30,                     # Slightly larger max size for animation visibility
    zoom=2,                          # 📝 TO DO: Adjust zoom for your region
    title="Institution Reddit Posts: Cumulative Location Mentions & Rolling Average Sentiment Over Time",
    center=dict(lat=35.5, lon=-80),  # 📝 TO DO: Center on your institution's area
    range_color=[-0.5, 0.5]          # Fixed color range for consistent comparison across time
)

# STEP 3: Customize animation settings and layout
fig_animated.update_layout(
    map_style="carto-positron",
    width=800,
    height=600,
    title_font_size=16,
    title_x=0.5,
    coloraxis_colorbar=dict(         # Customize the sentiment legend
        title="Rolling Avg<br>Sentiment",
        tickmode="linear",
        tick0=-0.5,                  # Start legend at -0.5 (most negative)
        dtick=0.25                   # Tick marks every 0.25 points
    )
)

# STEP 4: Set animation timing (in milliseconds)
# 📝 TO DO: Experiment with these values for optimal viewing
fig_animated.layout.updatemenus[0].buttons[0].args[1]["frame"]["duration"] = 800    # Time between frames
fig_animated.layout.updatemenus[0].buttons[0].args[1]["transition"]["duration"] = 300 # Transition smoothness

# Display with HTML export configuration
fig_animated.show(config={'displayModeBar': True, 'displaylogo': False})

	school_name	unique_id	date	sentences	roberta_compound	place	latitude	longitude	revised_place	revised_latitude	revised_longitude	place_type	false_positive	checked_by
0	UNC	UNC_10299	2021-04-17 18:15:17	Missing all of South Campus & all of the apart...	-0.674987	501	38.68287	16.10256	501	38.68287	16.10256	Unknown	<NA>	Not Checked
1	UNC	UNC_9515	2024-08-27 20:37:43	Not sure if it was a student or resident but e...	-0.601315	Abernathy	33.83230	-101.84295	Abernathy	33.83230	-101.84295	Unknown	<NA>	Not Checked
2	UNC	UNC_1986	2024-04-29 00:02:28	Israel is a key figure for the United States i...	-0.026644	Al Bank al Brīţānī lish Sharq al Awsaţ	25.26430	55.29326	Al Bank al Brīţānī lish Sharq al Awsaţ	25.26430	55.29326	Unknown	<NA>	Not Checked
3	UNC	UNC_1987	2024-04-29 00:02:28	Iranian Parliament members came together and c...	-0.387580	Al Bank al Brīţānī lish Sharq al Awsaţ	25.26430	55.29326	Al Bank al Brīţānī lish Sharq al Awsaţ	25.26430	55.29326	Unknown	<NA>	Not Checked
4	UNC	UNC_3411	2020-08-27 06:13:30	Alabama pulled ahead with 531 in a week.	0.051380	Alabama	32.75041	-86.75026	Alabama	32.75041	-86.75026	State	<NA>	Not Checked

Project 4: Emotions of JMU¶

Understanding the Place where we Study¶

[Names]¶

📚 Table of Contents¶

📋 Getting Started¶

🔍 Data Exploration¶

🧹 Data Processing¶

🗺️ Spatial Analysis¶

😊 Sentiment Analysis¶

⏰ Time Series Analysis¶

📝 Final Report¶

📋 Assignment Overview¶

📝 Instructions¶

⏰ Timeline¶

Part 1: Research Overview¶

Introduction¶

1.1 Corpus Description¶

📊 Action Items¶

✍️ Writing Task¶

1.2 Hypothesis¶

✍️ Writing Task¶

💡 Sample Hypotheses¶

Part 2: Exploratory Data Analysis¶

📊 Action Items¶

✍️ Writing Task¶

2.1 Visualization 1 [Change Title Depending on Visualization]¶

✍️ Writing Task¶

📊 Add Visualization 1 Here¶

✍️ Visualization Analysis¶

2.2 Visualization 2 [Change Title Depending on Visualization]¶

✍️ Writing Task¶

Visualization Analysis¶

Part 3: Data Cleaning and Refinement¶

📋 About _processed.csv Files¶

3.1 Toponym Misalignment Analysis¶

📊 Action Items¶

✍️ Writing Task¶

3.2 Toponym Refinement¶

🔧 Step 1: Access Your Data¶

📋 Step 2: Data Structure¶

🔧 Step 3: Data Cleaning Process¶

💾 Step 4: Export Clean Data¶

3.3 Revised Map¶

✍️ Writing Task¶

🔧 Technical Implementation¶

3.4 Map Customization¶

🔧 Map Adjustment Instructions¶

Revision Insights¶

Part 4: Spatial Comparison¶

🔧 Function Parameters¶

💡 Example Usage¶

🔧 Customization Instructions¶

4.1 JMU Spatial Distribution¶

4.2 Spatial Analysis¶

✍️ Writing Task¶

Part 5: Sentiment Analysis Comparison¶

🔧 Function Parameters¶

💡 Example Usage¶

🔧 Customization Instructions¶

Sentiment Comparison Analysis¶

Part 6: Time Series Animation Analysis¶

🔧 Customization Instructions¶

✍️ Time Series Analysis¶

Part 7: Conclusion and Future Research¶

✍️ Final Writing Task¶

📋 Required Elements¶

📋 About `_processed.csv` Files¶